from tools.spider_utils import *
from browser import Driver
from Tmall import Tmall
from config import Temp_Path


def broker(tmall, flag, keyword, pageSize):
    '''
    :param flag: 是否走本地获取
    :param p_size: 循环次数
    :return:
    '''
    res = []
    for i in range(1, pageSize):
        if flag:
            # 从网络获取
            ids = tmall.get_ids_url(Temp_Path, keyword, str(i))
            time.sleep(3)  # 随机休眠
        else:
            # 从本地获取
            ids = tmall.get_ids_file(Temp_Path, keyword, str(i))
        res.append(ids)
    return res


def main():
    driver = Driver()
    t_mall = Tmall(driver)
    try:
        for i in range(3):  # task_id ->
            item_list = broker(t_mall, 1, "关键字", 20)
            for items in item_list:
                # check 检测当前数据库是否有重复
                res=is_data_exists(i,items)
                for (flag ,i)in res:
                    if not flag:
                        # todo 标记删除重复数据
                        items.remove(i)
                 # todo 添加任务队列
                add_task(i,items)

    except Exception as e:
        # todo 处理失败重做逻辑
        print(e)
        print("爬虫出现异常")
    finally:
        pass
